package com.sena.search;

import com.alibaba.fastjson.JSON;
import com.sena.crawler.SenaCrawler;
import com.sena.crawler.domian.RootLink;
import com.sena.crawler.enums.LinkTypeEnums;
import com.sena.crawler.loader.strategy.HtmlUnitPageLoader;
import com.sena.crawler.util.JedisUtilNew;
import com.sena.domain.SenaClientSite;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * @author seven
 * @create 2021-08-10 16:48
 **/
public class SearchCommonLink {

    private static Logger logger = LoggerFactory.getLogger(SearchCommonLink.class);

    /**
     * 参看 Main 和builder.properties 中的配置
     *
     * @param site      站点信息
     * @param set       需要匹配的关键字
     * @param blackurls 黑名单
     * @param whiteurls 白名单
     * @param suffixs   过滤关键字
     */
    public void SearchCommonLinkYunTask(SenaClientSite site, Set<String> set, List<String> blackurls, List<String> whiteurls, List<String> suffixs) {
        //初始化url
        JedisUtilNew jedisUtil = new JedisUtilNew(site.getRedisDbAddr());
        SenaCrawler crawler = new SenaCrawler.Builder()
                .setSenaCrawlerType(LinkTypeEnums.COMMON_LINK.getCode())
                .setRootUrls(site.getSiteAddr())//设置根节点
                //设置白名单
                .setWhiteUrlRegexs(whiteurls)
                //设置黑名单
                .setBlackUrlRegexs(blackurls)
                .setPageLoader(new HtmlUnitPageLoader())
                .setThreadCount(site.getThreadNum())
                .setAllowSpread(true)//设置是否允许扩散爬取
                .setFilterSuffix(suffixs) //拦截后缀
                //设置是否进行正则匹配
                .setAllowRegex(site.getIsNeedRegex()==1?true:false)
                //爬取关键字页面
                .setFilterKeyWorld(set)
                .setJdbcUrl("jdbc:mysql://" + site.getMysqlDbIp() + ":" + site.getMysqlDbIp() + "/" + site.getMysqlDbName() + "?serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8")
                .setJdbcUserName("root")
                .setJdbcUserPassWord(site.getMysqlDbPassword())
                .setJedisUtil(jedisUtil)
                .setMixUrlCount(site.getMixUrlCount())
                .setNeedMax(site.getIsNeedMax() == 1 ? true : false)
                .build();

        init(jedisUtil, site.getSiteAddr());
        // 启动
        crawler.start(true);
    }

    /**
     * 参看 Main 和builder.properties 中的配置
     *
     * @param address   redis 地址
     * @param set       需要匹配的关键字
     * @param rootUrl   根节点
     * @param blackurls 黑名单
     * @param whiteurls 白名单
     */
    public void SearchCommonLinkTask(String address,
                                     Set<String> set,
                                     String rootUrl,
                                     List<String> blackurls,
                                     List<String> whiteurls,
                                     List<String> suffixs) {
        //初始化url

        JedisUtilNew jedisUtil = new JedisUtilNew(address);
        SenaCrawler crawler = new SenaCrawler.Builder()
                .setSenaCrawlerType(LinkTypeEnums.COMMON_LINK.getCode())
                .setRootUrls(rootUrl)
                //设置白名单
                .setWhiteUrlRegexs(whiteurls)
                //设置黑名单
                .setBlackUrlRegexs(blackurls)
                .setPageLoader(new HtmlUnitPageLoader())
                .setThreadCount(1)
                .setAllowSpread(true)
                .setFilterSuffix(suffixs)
                .setFilterKeyWorld(set)
                .setJedisUtil(jedisUtil)
                .setMixUrlCount(10)
                .setNeedMax(true)
                //设置是否进行正则匹配
                .setAllowRegex(true)
                .build();
        // 启动
        crawler.start(true);
        crawler.tryFinish();
    }


    /**
     * 保存通用url
     */
    public static void init(JedisUtilNew jedisUtil, String address) {
        //JedisUtil.init("redis://127.0.0.1:6379/2");
        // 设置根节点
        RootLink rootLink = new RootLink(address, false, null, null);
        //设置根节点
        jedisUtil.setStringValue(LinkTypeEnums.ROOT_LINK.getCode(), JSON.toJSONString(rootLink));

    }

    public static void main(String[] args) {
        Set<String> strings = new HashSet<>();
        List<String> list = new ArrayList<>();
        list.add(".*hncpu.*");
        list.add("machinery.ccsu.xk.hnlat.com/");
        List<String> suffixs = new ArrayList<>();
        SearchCommonLink searchCommonLink = new SearchCommonLink();
        searchCommonLink.SearchCommonLinkTask("redis://127.0.0.1:6379/0", strings,
                "http://machinery.ccsu.xk.hnlat.com/", null, list, suffixs);
        System.out.println("111");
    }

}
